import scrapy
import json
import re


class Products_Spider(scrapy.Spider):
    # 读取所有url
    file = open(
        '../../data/products_url', 'r')
    url_list = []
    for line in file:
        url_list.append(line)
    name = "products"
    allowed_domains = ["list.jd.com"]
    # start_urls = [
    #     "https://list.jd.com/list.html?tid=1001865%0A",
    # ]
    start_urls = url_list

    def parse(self, response):

        products_category = response.css(
            '.s-title h3 b').xpath('text()').extract()[0]

        result = []
        for sel in response.xpath("//div[@class='selector']"):
            # # 获取品牌
            for sel_brand in sel.css('.sl-v-logos'):
                attr = {}
                key = '品牌'
                value = sel_brand.css('ul li a').xpath('@title').extract()
                attr['name'] = key
                attr['value_name'] = value
                if value:
                    result.append(attr)
            # 获取除品牌外的其他内容
            for sel_wrap in sel.css('.sl-wrap'):
                attr = {}
                key = sel_wrap.css('.sl-key span').xpath('text()').extract()[0]
                value = sel_wrap.css(
                    '.sl-v-list ul li a').xpath('text()').extract()
                attr['name'] = key
                attr['value_name'] = value
                if value:
                    result.append(attr)

        # 通过获取json数据
        # 读取respongse中other_exts中的内容
        res = response.body_as_unicode()
        start = res.find('other_exts')
        end = res.find('var pay_after')
        pre_result = res[start: end]

        # 如果获取到的pre_result太短，说明里面没有内容，则不进行处理
        if len(pre_result) > 35:
            other_attr = str(re.findall(r'{.*}', pre_result))
            # 将other_exts中多余的属性去掉
            other_attr = other_attr[3: -3]
            split_arr = other_attr.split('},{')
            for string in split_arr:
                str_arr = string.split(',')
                split_str = '{' + str_arr[0] + ',' + str_arr[-1] + '}'
                # 将去掉多余属性的项转换成字典形式
                split_dict = eval(split_str)
                value_name_str = split_dict['value_name']
                new_dict = {}
                new_dict['name'] = split_dict['name']
                new_dict['value_name'] = value_name_str.split(';')
                result.append(new_dict)
        # 将结果转换成字典形式
        result_dict = {}
        result_dict[products_category] = result
        # 去掉数据中原有的空格，并转成json
        result_str = str(result_dict).replace(' ', '')
        result_str = result_str.replace('：', '')
        # jo = json.dumps(result_str)
        with open("../../data/products-data", "a") as f:
            #     # json.dump(jo, f)
            f.write(result_str + '\n')
